--- /dev/null
+From: Carbo Kuo <byvoid@byvoid.com>
+Date: Thu, 25 Feb 2021 21:13:38 +0900
+Subject: Fix a severe performance bug in `Conversion::Convert` that caused
+ O(N^2) complexity.
+
+In `Conversion.cpp`, line 27:
+```
+ Optional<const DictEntry*> matched = dict->MatchPrefix(pstr);
+```
+pstr is a `const char*`. However, there is no overloaded function which parameter `const char*`.
+Therefore it matches `Optional<const DictEntry*> MatchPrefix(const std::string& word) const`.
+There is an implicit type conversion from `char*` to `std::string` with time complexity O(N).
+
+I added new benchmark tests. Before the fix:
+
+1: ------------------------------------------------------------------
+1: Benchmark Time CPU Iterations
+1: ------------------------------------------------------------------
+1: BM_Initialization/hk2s 1.17 ms 1.12 ms 645
+1: BM_Initialization/hk2t 0.116 ms 0.116 ms 5922
+1: BM_Initialization/jp2t 0.206 ms 0.201 ms 3500
+1: BM_Initialization/s2hk 18.2 ms 17.9 ms 40
+1: BM_Initialization/s2t 18.2 ms 18.1 ms 39
+1: BM_Initialization/s2tw 17.9 ms 17.8 ms 39
+1: BM_Initialization/s2twp 18.6 ms 18.4 ms 39
+1: BM_Initialization/t2hk 0.055 ms 0.054 ms 12907
+1: BM_Initialization/t2jp 0.120 ms 0.117 ms 5978
+1: BM_Initialization/t2s 0.988 ms 0.984 ms 710
+1: BM_Initialization/tw2s 1.08 ms 1.05 ms 672
+1: BM_Initialization/tw2sp 1.26 ms 1.24 ms 563
+1: BM_Initialization/tw2t 0.088 ms 0.083 ms 8528
+1: BM_Convert2M 413 ms 413 ms 2
+1: BM_Convert/100 1.09 ms 1.09 ms 629
+1: BM_Convert/1000 33.2 ms 33.2 ms 21
+1: BM_Convert/10000 2822 ms 2822 ms 1
+1: BM_Convert/100000 (took longer than 5 minutes, killed)
+
+Now:
+1: ------------------------------------------------------------------
+1: Benchmark Time CPU Iterations
+1: ------------------------------------------------------------------
+1: BM_Initialization/hk2s 1.07 ms 1.07 ms 650
+1: BM_Initialization/hk2t 0.114 ms 0.114 ms 6092
+1: BM_Initialization/jp2t 0.204 ms 0.200 ms 3503
+1: BM_Initialization/s2hk 18.2 ms 18.0 ms 40
+1: BM_Initialization/s2t 17.6 ms 17.6 ms 39
+1: BM_Initialization/s2tw 18.0 ms 17.9 ms 40
+1: BM_Initialization/s2twp 17.9 ms 17.9 ms 39
+1: BM_Initialization/t2hk 0.055 ms 0.055 ms 12855
+1: BM_Initialization/t2jp 0.125 ms 0.124 ms 5772
+1: BM_Initialization/t2s 1.000 ms 0.989 ms 695
+1: BM_Initialization/tw2s 1.09 ms 1.07 ms 668
+1: BM_Initialization/tw2sp 1.29 ms 1.26 ms 558
+1: BM_Initialization/tw2t 0.082 ms 0.082 ms 8558
+1: BM_Convert2M 361 ms 361 ms 2
+1: BM_Convert/100 0.741 ms 0.740 ms 948
+1: BM_Convert/1000 7.54 ms 7.52 ms 94
+1: BM_Convert/10000 76.3 ms 76.3 ms 9
+1: BM_Convert/100000 786 ms 786 ms 1
+
+This bug was reported in https://github.com/BYVoid/OpenCC/issues/478 and https://github.com/BYVoid/OpenCC/issues/517.
+
+Applied-Upstream: https://github.com/BYVoid/OpenCC/commit/c2e548e5e95c9a8ccc5c1e5feb259e8885ef32c6
+---
+ src/Dict.hpp | 7 ++++
+ src/benchmark/Performance.cpp | 77 ++++++++++++++++++++++++++++++++++---------
+ 2 files changed, 69 insertions(+), 15 deletions(-)
+
+diff --git a/src/Dict.hpp b/src/Dict.hpp
+index 461d6d2..1c81034 100644
+--- a/src/Dict.hpp
++++ b/src/Dict.hpp
+@@ -49,6 +49,13 @@ public:
+ virtual Optional<const DictEntry*> MatchPrefix(const char* word,
+ size_t len) const;
+
++ /**
++ * Matches the longest matched prefix of a word.
++ */
++ Optional<const DictEntry*> MatchPrefix(const char* word) const {
++ return MatchPrefix(word, KeyMaxLength());
++ }
++
+ /**
+ * Matches the longest matched prefix of a word.
+ */
+diff --git a/src/benchmark/Performance.cpp b/src/benchmark/Performance.cpp
+index cf8d3aa..d1b6468 100644
+--- a/src/benchmark/Performance.cpp
++++ b/src/benchmark/Performance.cpp
+@@ -1,7 +1,26 @@
++/*
++ * Open Chinese Convert
++ *
++ * Copyright 2020-2021 Carbo Kuo <byvoid@byvoid.com>
++ *
++ * Licensed under the Apache License, Version 2.0 (the "License");
++ * you may not use this file except in compliance with the License.
++ * You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
+ #include <benchmark/benchmark.h>
+ #include <fstream>
+ #include <iostream>
+ #include <memory>
++#include <sstream>
+ #include <streambuf>
+
+ #ifdef _MSC_VER
+@@ -44,21 +63,32 @@ static void BM_Initialization(benchmark::State& state,
+ state.ResumeTiming();
+ }
+ }
+-BENCHMARK_CAPTURE(BM_Initialization, hk2s, "hk2s");
+-BENCHMARK_CAPTURE(BM_Initialization, hk2t, "hk2t");
+-BENCHMARK_CAPTURE(BM_Initialization, jp2t, "jp2t");
+-BENCHMARK_CAPTURE(BM_Initialization, s2hk, "s2hk");
+-BENCHMARK_CAPTURE(BM_Initialization, s2t, "s2t");
+-BENCHMARK_CAPTURE(BM_Initialization, s2tw, "s2tw");
+-BENCHMARK_CAPTURE(BM_Initialization, s2twp, "s2twp");
+-BENCHMARK_CAPTURE(BM_Initialization, t2hk, "t2hk");
+-BENCHMARK_CAPTURE(BM_Initialization, t2jp, "t2jp");
+-BENCHMARK_CAPTURE(BM_Initialization, t2s, "t2s");
+-BENCHMARK_CAPTURE(BM_Initialization, tw2s, "tw2s");
+-BENCHMARK_CAPTURE(BM_Initialization, tw2sp, "tw2sp");
+-BENCHMARK_CAPTURE(BM_Initialization, tw2t, "tw2t");
++BENCHMARK_CAPTURE(BM_Initialization, hk2s, "hk2s")
++ ->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Initialization, hk2t, "hk2t")
++ ->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Initialization, jp2t, "jp2t")
++ ->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Initialization, s2hk, "s2hk")
++ ->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Initialization, s2t, "s2t")->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Initialization, s2tw, "s2tw")
++ ->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Initialization, s2twp, "s2twp")
++ ->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Initialization, t2hk, "t2hk")
++ ->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Initialization, t2jp, "t2jp")
++ ->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Initialization, t2s, "t2s")->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Initialization, tw2s, "tw2s")
++ ->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Initialization, tw2sp, "tw2sp")
++ ->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Initialization, tw2t, "tw2t")
++ ->Unit(benchmark::kMillisecond);
+
+-static void BM_Convert(benchmark::State& state) {
++static void BM_Convert2M(benchmark::State& state) {
+ const std::string config_name = "s2t";
+ const std::string text = ReadText("zuozhuan.txt");
+ const std::unique_ptr<SimpleConverter> converter(Initialize(config_name));
+@@ -66,7 +96,24 @@ static void BM_Convert(benchmark::State& state) {
+ Convert(converter.get(), text);
+ }
+ }
+-BENCHMARK(BM_Convert)->Unit(benchmark::kMillisecond);
++BENCHMARK(BM_Convert2M)->Unit(benchmark::kMillisecond);
++
++static void BM_Convert(benchmark::State& state, int iteration) {
++ std::ostringstream os;
++ for (int i = 0; i < iteration; i++) {
++ os << "Open Chinese Convert 開放中文轉換" << i << std::endl;
++ }
++ const std::string text = os.str();
++ const std::string config_name = "s2t";
++ const std::unique_ptr<SimpleConverter> converter(Initialize(config_name));
++ for (auto _ : state) {
++ Convert(converter.get(), text);
++ }
++}
++BENCHMARK_CAPTURE(BM_Convert, 100, 100)->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Convert, 1000, 1000)->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Convert, 10000, 10000)->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Convert, 100000, 100000)->Unit(benchmark::kMillisecond);
+
+ } // namespace opencc
+
projects / opencc.git / commitdiff